//+---------------------------------------------------------------------------
//
// Copyright (C) Microsoft Corporation, 2008
//
// File: DeliveryService.cs
//
// Contents: The Delivery Service, WCF web service - implementation of IDeliveryService contract
//
// Created by: Adam Wardynski (adamwa) Feb 13 2008
//
//----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data.Linq;
using System.Linq;
using System.Reflection;
using System.ServiceModel;
using System.ServiceModel.Channels;
using Microsoft.MSAS.ManagedAPI;
using MRS.FrontEnd.BO;

namespace Microsoft.Reputation.Services
{
    [ServiceBehavior(Namespace = "http://schemas.microsoft.com/MRS/Services/2009/07/01")]
    public class MrsDeliveryService : IMrsDeliveryService
    {
        public const uint DefaultTelemetry = 0;
        public const uint DefaultDelayCommMins = 0;
        public const long InvalidOverridesStamp = -1;

        private static bool s_isInitialized = false;
        private static object s_initializationSyncRoot = new object();
        //
        // Components used by DeliveryService
        //
        internal static IDataProvider DataProvider { get; set; }
        internal static IWebServicesApiLogger ApiLogger { get; set; }
        //
        // Allowed sizes for request parameters read from config file
        //
        internal static int MaxLookupRequestEntriesCount { get; set; }
        internal static int MaxListingRequestEntriesCount { get; set; }
        private static int s_listingPageSize;
        private static uint s_delayCommunicationMinutes;
        internal static bool s_lookupCategoriesForUrlApiEnabled;
        private static Guid s_AllowedLookupUrlId;

        /// <summary>
        /// This is the last override stamp that we have gotten back from the
        /// backend.  WARNING: This value should only be set by the OverrideThreadObject
        /// It is ok to get the value, but any sets of the value need to go through 
        /// strict locking to keep things behaving correctly
        /// </summary>
        internal static long LastKnownOverrideStamp { get; set; }

        /// <summary>
        /// This list is of the overrides we have already gotten from the back end.
        /// Set calls should only be done by the OverrideThreadObject.
        /// </summary>
        internal static SortedList<long, byte[]> OverrideHashesFromBackEnd;

        /// <summary>
        /// The maximum overrides that are allowed to be put into a single Lookup Response 
        /// package sent back to partners
        /// </summary>
        internal static int MaxOverridesPerPacket { get; set; }

        internal static bool IsInitialized
        {
            get
            {
                return s_isInitialized;
            }
            set
            {
                s_isInitialized = value;
            }
        }

        #region Initialize and Shutdown logic

        public static void Initialize()
        {
            TraceUtility.TraceHeader(Tracing.Source);

            using (TraceUtility.TraceMethod(Tracing.Source))
            {
                lock (s_initializationSyncRoot)
                {
                    if (MrsDeliveryService.IsInitialized)
                    {
                        TraceUtility.TraceWarning(Tracing.Source, "Initialize called for already initialized DeliveryService");
                    }
                    else
                    {
                        EventLogger.Initialize();
                        EventLogger.LogServiceStartingEvent();

                        PerfCounters.Initialize();

                        MrsDeliveryService.DataProvider = CreateDataProvider();
                        MrsDeliveryService.ApiLogger = CreateApiLogger();

                        MrsDeliveryService.MaxLookupRequestEntriesCount = DeliveryServiceConfig.MaxLookupRequestEntriesCount;
                        MrsDeliveryService.MaxListingRequestEntriesCount = DeliveryServiceConfig.MaxListingRequestEntriesCount;
                        MrsDeliveryService.MaxOverridesPerPacket = DeliveryServiceConfig.MaxOverridesPerPacket;
                        s_listingPageSize = DeliveryServiceConfig.ListingPageSize;
                        s_delayCommunicationMinutes = DeliveryServiceConfig.DelayCommunicationMinutes;
                        s_lookupCategoriesForUrlApiEnabled = DeliveryServiceConfig.LookupCategoriesForUrlApiEnabled;
                        s_AllowedLookupUrlId = new Guid(DeliveryServiceConfig.AllowedLookupUrlId);

                        // The first valid override stamp will be two.  We need to use 1 to trigger our partners that
                        // they have the latest override stamp so that when they send it up we can give them any 
                        // overrides beyond that.  This get the whole chain initialized correctly.
                        MrsDeliveryService.LastKnownOverrideStamp = 1;
                        OverrideHashesFromBackEnd = new SortedList<long, byte[]>();

                        MrsDeliveryService.IsInitialized = true;

                        EventLogger.LogServiceStartedEvent();
                    }
                }
            }
        }

        public static void Shutdown()
        {
            using (TraceUtility.TraceMethod(Tracing.Source))
            {
                lock (s_initializationSyncRoot)
                {
                    TraceUtility.TraceAssert(Tracing.Source, s_isInitialized, "Shutdown called for unitiliazied DeliveryService");
                    if (MrsDeliveryService.IsInitialized)
                    {
                        TraceUtility.TraceInfo(Tracing.Source, "Shutdown at {0}", DateTime.Now);
                        EventLogger.LogServiceStoppingEvent();

                        MrsDeliveryService.IsInitialized = false;
                        try
                        {
                            using (MrsDeliveryService.DataProvider as IDisposable)
                            {
                                using (MrsDeliveryService.ApiLogger as IDisposable)
                                {
                                    MrsDeliveryService.ApiLogger.StopLogging();
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            TraceUtility.TraceThrowException(Tracing.Source, ex);
                            TraceUtility.TraceAssert(Tracing.Source, false, "Exception in shutdown logic");
                            TraceUtility.TraceThrowException(Tracing.Source, ex);
                            throw;
                        }
                        finally
                        {
                            MrsDeliveryService.ApiLogger = null;

                            MrsDeliveryService.MaxLookupRequestEntriesCount = DeliveryServiceConfig.MaxLookupRequestEntriesCountDefault;
                            MrsDeliveryService.MaxListingRequestEntriesCount = DeliveryServiceConfig.MaxListingRequestEntriesCountDefault;
                            MrsDeliveryService.MaxOverridesPerPacket = DeliveryServiceConfig.MaxOverridesPerPacket;
                            s_listingPageSize = DeliveryServiceConfig.ListingPageSizeDefault;
                            s_delayCommunicationMinutes = DeliveryServiceConfig.DelayCommunicationMinutesDefault;
                            s_lookupCategoriesForUrlApiEnabled = DeliveryServiceConfig.LookupCategoriesForUrlApiEnabled;
                            s_AllowedLookupUrlId = new Guid(DeliveryServiceConfig.AllowedLookupUrlIdDefault);

                            PerfCounters.Shutdown();

                            EventLogger.LogServiceStoppedEvent();
                            EventLogger.Shutdown();
                        }
                    }
                }
            }
        }

        private static IDataProvider CreateDataProvider()
        {
            using (TraceUtility.TraceMethod(Tracing.Source))
            {
                Type theDataProvider = null;
                IDataProvider dp = null;
                try
                {
                    if (DeliveryServiceConfig.ReputationCacheEnabled)
                    {
                        theDataProvider = Type.GetType(DeliveryServiceConfig.CacheDataProviderTypeDefault);
                    }
                    else
                    {
                        theDataProvider = Type.GetType(DeliveryServiceConfig.DataProviderTypeDefault);
                    }

                    if (theDataProvider != null)
                    {
                        ConstructorInfo constructor = theDataProvider.GetConstructor(Type.EmptyTypes);
                        if (constructor != null)
                        {
                            dp = constructor.Invoke(new object[0]) as IDataProvider;
                        }
                    }
                }
                catch (Exception ex)
                {
                    TraceUtility.TraceCatchException(Tracing.Source, ex);
                    TraceUtility.TraceAssert(Tracing.Source, false, "Failed to initialize specified IDataProvider");
                    TraceUtility.TraceThrowException(Tracing.Source, ex);
                    throw;
                }

                return dp;
            }
        }

        internal static IWebServicesApiLogger CreateApiLogger()
        {
            using (TraceUtility.TraceMethod(Tracing.Source))
            {
                try
                {
                    if (DeliveryServiceConfig.ApiLoggingTurnedOff)
                    {
                        return new EmptyApiLogger();
                    }
                    return new WebServicesApiLogger();
                }
                catch (Exception ex)
                {
                    TraceUtility.TraceCatchException(Tracing.Source, ex);
                    TraceUtility.TraceAssert(Tracing.Source, false, "Failed to initialize WebServicesApiLogger");
                    TraceUtility.TraceThrowException(Tracing.Source, ex);
                    throw;
                }
            }
        }

        #endregion

        #region Lookup generic handling

        internal static void GetLookupResult(IAsyncResult asyncResult)
        {
            using (TraceUtility.TraceMethod(Tracing.Source))
            {
                LookupAsyncResult ar = null;
                List<LookupResponse> result = null;
                Exception exception = null;

                try
                {
                    if (asyncResult == null || asyncResult.AsyncState == null)
                    {
                        throw new ArgumentNullException("asyncResult or asyncResult.AsyncState");
                    }
                    else
                    {

                        ar = (LookupAsyncResult)asyncResult.AsyncState;

                        DPLookupResponse[] responseArray = DataProvider.EndLookupObjects(
                            out ar.OwnState.Status,
                            out ar.OwnState.DataProviderLatency,
                            asyncResult);

                        result = DPLookupResponse.ConvertLookupToLookupResponses(responseArray, ar.OwnState.Requests);

                        EventLogger.LogTopLevelExceptionThreadSuccess();
                    }
                }
                catch (Exception ex)
                {
                    if (ex is LoggedException)
                    {
                        EventLogger.LogTopLevelExceptionThreadSuccess();
                    }
                    else
                    {
                        EventLogger.LogTopLevelExceptionThreadFailure(ex.ToString());
                        TraceUtility.TraceCatchException(Tracing.Source, ex);
                    }

                    exception = ex;
                }
                finally
                {
                    if (ar != null)
                    {
                        if (exception == null)
                        {
                            ar.SetComplete(result, asyncResult.CompletedSynchronously);
                        }
                        else
                        {
                            ar.SetComplete(exception, asyncResult.CompletedSynchronously);
                        }
                    }
                }
            }
        }

        public IAsyncResult BeginLookupObjects(
            ClientIdToken clientId,
            LookupRequest[] requests,
            Filter filter,
            StateToken stateToken,
            long latestOverrideStamp,
            AsyncCallback callback,
            object state)
        {
            PerfCounters.LookupObjectsApiCalled();
            string clientIpAddress = null;

            try
            {
                OperationContext context = OperationContext.Current;
                MessageProperties messageProperties = context.IncomingMessageProperties;

                RemoteEndpointMessageProperty endpointProperty =
                    messageProperties[RemoteEndpointMessageProperty.Name] as RemoteEndpointMessageProperty;

                clientIpAddress = endpointProperty.Address;
            }
            catch
            {
                // Cannot do much if we cannot obtain the client IP address.
            }

            LookupState lookupState = new LookupState(
                clientId,
                requests,
                filter,
                stateToken,
                clientIpAddress);

            LookupAsyncResult asyncResult = new LookupAsyncResult(lookupState, callback, state);
            //
            // current assumption is to ignore invalid stateToken and just new up one
            //
            if (stateToken == null || !stateToken.IsValid())
            {
                lookupState.ResponseStateToken = new StateToken();
            }
            else
            {
                lookupState.ResponseStateToken = new StateToken(lookupState.RequestStateToken);
            }

            try
            {
                uint status = AreArgumentsValid(clientId, requests, MrsDeliveryService.MaxLookupRequestEntriesCount, latestOverrideStamp);

                if (status != StatusCodes.OK)
                {
                    TraceUtility.TraceError(
                        Tracing.Source,
                        "Invalid arguments for LookupObjects");
                    lookupState.Status = status;
                    asyncResult.SetComplete(null, true);
                    return asyncResult;
                }

                status = TypeSpecificValidation.IsRequestValid(requests, filter);

                if (status != StatusCodes.OK)
                {
                    TraceUtility.TraceError(
                        Tracing.Source,
                        "Invalid type specific parameters for LookupObjects");
                    lookupState.Status = status;
                    asyncResult.SetComplete(null, true);
                    return asyncResult;
                }
            }
            catch (Exception ex)
            {
                TraceUtility.TraceCatchException(Tracing.Source, ex);
                lookupState.Status = StatusCodes.Error;
                asyncResult.SetComplete(ex, true);
                return asyncResult;
            }

            try
            {
                LookupRequest[] lookupRequests = requests;

                if (latestOverrideStamp != 0
                    && latestOverrideStamp < MrsDeliveryService.LastKnownOverrideStamp)
                {
                    lookupRequests = AddOverridesRequest(lookupRequests, ref latestOverrideStamp);
                    lookupState.LatestOverridesStamp = latestOverrideStamp;
                    lookupState.Requests = lookupRequests;
                }
                else
                {
                    lookupState.LatestOverridesStamp = MrsDeliveryService.LastKnownOverrideStamp;
                }


                DataProvider.BeginLookupObjects(
                    DPLookupRequest.ConvertToDPLookupRequests(lookupRequests),
                    GetLookupResult,
                    asyncResult);

            }
            catch (Exception ex)
            {
                TraceUtility.TraceCatchException(Tracing.Source, ex);
                lookupState.Status = StatusCodes.Error;
                asyncResult.SetComplete(ex, true);
            }
            return asyncResult;
        }

        public ICollection<LookupResponse> EndLookupObjects(
            out StateToken stateToken,
            out long latestOverrideStamp,
            out uint delayCommMins,
            out uint operStatus, 
            IAsyncResult asyncResult)
        {
            List<LookupResponse> result = null;
            operStatus = StatusCodes.OK;
            LookupState lookupState = null;
            LookupAsyncResult lookupAsyncResult = null;
            delayCommMins = s_delayCommunicationMinutes;
            Guid clientIdGuid = Guid.Empty;

            try
            {
                lookupAsyncResult = (LookupAsyncResult)asyncResult;
                lookupState = lookupAsyncResult.OwnState;

                result = lookupAsyncResult.EndInvoke();

                operStatus = lookupState.Status;

                if (lookupState.ClientId != null)
                {
                    clientIdGuid = lookupState.ClientId.ClientId;
                }

                stateToken = lookupState.ResponseStateToken;
                latestOverrideStamp = lookupState.LatestOverridesStamp;

                if (operStatus == StatusCodes.OK)
                {
                    if (lookupState.Filter != null)
                    {
                        foreach (LookupResponse response in result)
                        {
                            // Filter the data based on the Global Filter
                            List<ReputationObject> filteredObjects = new List<ReputationObject>();

                            foreach (ReputationObject repuObj in response.Matches)
                            {
                                filteredObjects.Add(lookupState.Filter.FilterOut(repuObj));
                            }
                            response.Matches = filteredObjects;
                        }
                    }

                    // Build a copy of the response list to 
                    // work around a memory heap corruption issue
                    // that blows up the service.  Bug 1164
                    result = BuildCopyOfResponseMatches(result);

                    EventLogger.LogTopLevelExceptionThreadSuccess();

                    try
                    {
                        MrsDeliveryService.ApiLogger.LogLookupObjects(
                            lookupState.ClientId,
                            lookupState.Requests,
                            lookupState.Filter,
                            lookupState.RequestStateToken,
                            result,
                            stateToken,
                            latestOverrideStamp,
                            delayCommMins,
                            operStatus,
                            lookupState.ClientIpAddress);
                    }
                    catch (Exception ex)
                    {
                        TraceUtility.TraceCatchException(Tracing.Source, ex);

                        //
                        // Ignore the API Logger exception since it is not fatal.
                        //
                    }
                }
            }
            catch (Exception ex)
            {
                operStatus = StatusCodes.Error;
                latestOverrideStamp = InvalidOverridesStamp;
                result = null;
                if (lookupState != null)
                {
                    stateToken = lookupState.RequestStateToken;
                }
                else
                {
                    stateToken = null;
                    delayCommMins = 0;
                }
                if (DeliveryServiceHelpers.IsFatalException(ex))
                {
                    throw;
                }
                EventLogger.LogTopLevelExceptionThreadFailure(ex.ToString());
                TraceUtility.TraceCatchException(Tracing.Source, ex);
            }
            finally
            {
                PerfCounters.LookupObjectsApiCompleted(lookupAsyncResult.StopWatchTicks, operStatus);

                TraceUtility.TraceInfo(
                    Tracing.Source,
                    " [{0}] [PERF] EndLookupObjects [{1} ms] [{2} ms] {3} {4}",
                    clientIdGuid.ToString(),
                    lookupAsyncResult == null ? "null" : lookupAsyncResult.StopWatchMilliSec.ToString(),
                    lookupAsyncResult == null ? "null" : lookupAsyncResult.OwnState.DataProviderLatency.ToString(),
                    lookupState == null ? "null" : lookupState.Status.ToString(),
                    result == null ? "null" : result.Count.ToString());               
            }

            return result;
        }

        #endregion

        #region Listing generic handling

        public IAsyncResult BeginListRelatedObjects(
            ClientIdToken clientId,
            ListingRequest[] requests,
            Filter filter,
            StateToken stateToken,
            ListingToken listingToken,
            AsyncCallback callback,
            object state)
        {
            ListingState listingState = new ListingState(
                    clientId,
                    requests,
                    filter,
                    stateToken,
                    listingToken);

            ListingAsyncResult asyncResult = new ListingAsyncResult(listingState, callback, state);

            TraceUtility.TraceError(
                Tracing.Source,
                "ListingObjects API is not supported.");

            listingState.Status = StatusCodes.Error;
            asyncResult.SetComplete(null, true);

            return asyncResult;
        }

        public ICollection<ListingResponse> EndListRelatedObjects(
            out StateToken stateToken,
            out ListingToken listingToken,
            out long latestOverrideStamp,
            out uint remaining,
            out uint delayCommMins,
            out uint operStatus,
            IAsyncResult asyncResult)
        {
            List<ListingResponse> result = null;
            operStatus = StatusCodes.OK;
            delayCommMins = s_delayCommunicationMinutes;
            ListingState listingState = null;
            ListingAsyncResult listingAsyncResult = null;
            Guid clientIdGuid = Guid.Empty;

            try
            {
                listingAsyncResult = (ListingAsyncResult)asyncResult;
                listingState = listingAsyncResult.OwnState;

                if (listingState.ClientId != null)
                {
                    clientIdGuid = listingState.ClientId.ClientId;
                }

                result = listingAsyncResult.EndInvoke();

                operStatus = listingState.Status;
                listingToken = listingState.ResponseListingToken;
                stateToken = listingState.ResponseStateToken;
                latestOverrideStamp = listingState.LatestOverridesStamp;
                remaining = listingState.Remaining;

                //
                // The code below is never called because
                // The BeginListRelatedObjects always sets the 
                // operStatus to StatusCodes.Error

                //try
                //{
                //    if (operStatus == StatusCodes.OK)
                //    {
                //        ApiLogger.LogListRelatedObjects(
                //            listingState.ClientId,
                //            listingState.Requests,
                //            listingState.Filter,
                //            listingState.RequestStateToken,
                //            listingState.RequestListingToken,
                //            result,
                //            stateToken,
                //            listingToken,
                //            latestOverrideStamp,
                //            remaining,
                //            delayCommMins,
                //            operStatus);
                //    }
                //}
                //catch (Exception ex)
                //{
                //    TraceUtility.TraceCatchException(Tracing.Source, ex);

                //    //
                //    // Ignore the API Logger exception since it is not fatal.
                //    //
                //}
            }
            catch (Exception ex)
            {
                operStatus = StatusCodes.Error;
                remaining = 0;
                latestOverrideStamp = MrsDeliveryService.InvalidOverridesStamp;
                result = null;
                if (listingState != null)
                {
                    listingToken = listingState.RequestListingToken;
                    stateToken = listingState.RequestStateToken;
                }
                else
                {
                    listingToken = null;
                    stateToken = null;
                    delayCommMins = 0;
                }
                if (DeliveryServiceHelpers.IsFatalException(ex))
                {
                    throw;
                }
                TraceUtility.TraceCatchException(Tracing.Source, ex);                
            }
            finally
            {
                TraceUtility.TraceInfo(
                    Tracing.Source,
                    " [{0}] [PERF] EndListRelatedObjects [{1} ms] [{2} ms] {3} {4}",
                    clientIdGuid.ToString(),
                    listingAsyncResult == null ? "null" : listingAsyncResult.StopWatchMilliSec.ToString(),
                    listingAsyncResult == null ? "null" : listingAsyncResult.OwnState.DataProviderLatency.ToString(),
                    listingAsyncResult == null ? "null" : listingAsyncResult.OwnState.Status.ToString(),
                    result == null ? "null" : result.Count.ToString());
            }

            return result;
        }
        #endregion

        #region Lookup Categories For Url handling

        internal static void GetLookupUrlResult(IAsyncResult asyncResult)
        {
            using (TraceUtility.TraceMethod(Tracing.Source))
            {
                LookupUrlAsyncResult ar = null;
                List<CategoryReputation> result = null;
                Exception exception = null;

                try
                {
                    if (asyncResult == null || asyncResult.AsyncState == null)
                    {
                        throw new ArgumentNullException("asyncResult or asyncResult.AsyncState");
                    }
                    else
                    {

                        ar = (LookupUrlAsyncResult)asyncResult.AsyncState;

                        DPLookupResponse[] responseArray = DataProvider.EndLookupObjects(
                            out ar.OwnState.Status,
                            out ar.OwnState.DataProviderLatency,
                            asyncResult);

                        TraceUtility.TraceAssert(Tracing.Source, responseArray != null, "Response array expected not to be null MrsDeliveryService.GetLookupUrlResult");

                        if (ar.OwnState.Status == StatusCodes.OK
                            && responseArray != null
                            && responseArray.Length != 0)
                        {
                            if (responseArray.Length != 1)
                            {
                                TraceUtility.TraceError(
                                    Tracing.Source,
                                    "LookupCategoriesForUrl API got back {0} lookup responses from the database.  Should have only one response.",
                                    responseArray.Length);
                            }

                            SimpleReputationData[] simpleRepuData = CreateSimpleReputationDataFromResponses(responseArray[0]);

                            try
                            {
                                if (simpleRepuData.Length != 0)
                                {
                                    ICollection<SimpleCategory> simpleCategories =
                                        ar.OwnState.Categorizer.CategorizeResponseData(simpleRepuData);

                                    result = new List<CategoryReputation>(simpleCategories.Count);
                                    foreach (SimpleCategory sc in simpleCategories)
                                    {
                                        result.Add(new CategoryReputation(sc.categoryId, sc.confidence, null));
                                    }
                                    EventLogger.LogMrsCategorizeResponseApiSuccess();
                                }
                                else
                                {
                                    // Nothing to categorize so don't call the Client API as we are not
                                    // sure the behavior in that situation
                                    // To keep things easy on the consumer side of the API we send back an
                                    // empty list so that the caller can process the data without a null
                                    // check first
                                    result = new List<CategoryReputation>(0);                                    
                                }
                                EventLogger.LogTopLevelExceptionThreadSuccess();
                            }
                            catch (Exception ex1)
                            {
                                EventLogger.LogMrsCategorizeResponseApiFailure(
                                    string.Format("Failure categorizing url : {0} {1}", ar.OwnState.Url, ex1.Message));
                                // Re-throw so that we do not lose the context and 
                                // can deal with the exeception correctly on the return
                                throw;  
                            }

                        }
                    }
                }
                catch (Exception ex)
                {
                    EventLogger.LogTopLevelExceptionThreadFailure(ex.ToString());
                    TraceUtility.TraceCatchException(Tracing.Source, ex);
                    exception = ex;
                }
                finally
                {
                    if (ar != null)
                    {
                        if (exception == null)
                        {
                            ar.SetComplete(result, asyncResult.CompletedSynchronously);
                        }
                        else
                        {
                            ar.SetComplete(exception, asyncResult.CompletedSynchronously);
                        }
                    }
                }
            }
        }

        public IAsyncResult BeginLookupCategoriesForUrl(
            ClientIdToken clientId,
            string url,
            AsyncCallback callback,
            object state)
        {
            using (TraceUtility.TraceMethod(Tracing.Source))
            {
                PerfCounters.LookupCategoriesForUrlApiCalled();

                string clientIpAddress = null;
                try
                {
                    OperationContext context = OperationContext.Current;
                    MessageProperties messageProperties = context.IncomingMessageProperties;

                    RemoteEndpointMessageProperty endpointProperty =
                        messageProperties[RemoteEndpointMessageProperty.Name] as RemoteEndpointMessageProperty;

                    clientIpAddress = endpointProperty.Address;
                }
                catch
                {
                    // Cannot do much if we cannot obtain the client IP address.
                }

                LookupUrlState lookupUrlState = new LookupUrlState(clientId, clientIpAddress, url);
                LookupUrlAsyncResult asyncResult = new LookupUrlAsyncResult(lookupUrlState, callback, state);

                try
                {
                    uint status = MrsDeliveryService.AreArgumentsValid(clientId, url);

                    if (status != StatusCodes.OK)
                    {
                        TraceUtility.TraceError(
                            Tracing.Source,
                            "Invalid arguments for LookupCategoriesForUrl");
                        lookupUrlState.Status = status;
                        asyncResult.SetComplete(null, true);
                        return asyncResult;
                    }

                    if (s_lookupCategoriesForUrlApiEnabled == false
                        || s_AllowedLookupUrlId.Equals(clientId.CustomerId) == false)
                    {
                        TraceUtility.TraceError(
                            Tracing.Source,
                            "LookupCategoriesForUrl API is not supported.");

                        lookupUrlState.Status = StatusCodes.Error;
                        asyncResult.SetComplete(null, true);
                    }
                    else
                    {
                        byte[] datablock = null;
                        try
                        {
                            // Get the Datablock for the URL
                            ClientCategorizeProcessor categorizer = new ClientCategorizeProcessor(url);
                            lookupUrlState.Categorizer = categorizer;
                            datablock = categorizer.GetDataBlock();

                            EventLogger.LogMrsApiGetDataBlockSuccess();
                        }
                        catch (Exception ex1)
                        {
                            EventLogger.LogMrsApiGetDataBlockFailure(
                                string.Format("Failure getting datablock : {0} {1}", url, ex1.Message));
                            throw;
                        }

                        // if the datablock is null at this point we have other
                        // issues as the categorizer has failed to create the 
                        // datablock.  This should be handled by the throwing 
                        // of the exceptions above.
                        if (datablock != null)
                        {
                            List<DPLookupRequest> requests = new List<DPLookupRequest> {
                                new DPLookupUrlDataBlock { DataBlock = datablock }
                            };

                            DataProvider.BeginLookupObjects(
                                requests,
                                GetLookupUrlResult,
                                asyncResult);
                        }
                        else
                        {
                            lookupUrlState.Status = StatusCodes.Error;
                            asyncResult.SetComplete(null, true);
                        }
                    }
                }
                catch (Exception ex)
                {
                    TraceUtility.TraceCatchException(Tracing.Source, ex);
                    lookupUrlState.Status = StatusCodes.Error;
                    asyncResult.SetComplete(ex, true);
                }

                return asyncResult;
            }
        }

        public ICollection<CategoryReputation> EndLookupCategoriesForUrl(
            out uint operStatus,
            IAsyncResult asyncResult)
        {
            List<CategoryReputation> result = null;
            LookupUrlAsyncResult lookupUrlAsyncResult = null;
            LookupUrlState lookupUrlState = null;
            Guid clientIdGuid = Guid.Empty;

            try
            {
                lookupUrlAsyncResult = (LookupUrlAsyncResult)asyncResult;
                lookupUrlState = lookupUrlAsyncResult.OwnState;

                if (lookupUrlState.ClientId != null)
                {
                    clientIdGuid = lookupUrlState.ClientId.ClientId;
                }

                result = lookupUrlAsyncResult.EndInvoke();

                operStatus = lookupUrlState.Status;

                try
                {
                    if (operStatus == StatusCodes.OK)
                    {
                        MrsDeliveryService.ApiLogger.LogLookupCategoriesForUrl(
                            lookupUrlState.ClientId,
                            lookupUrlState.Url,
                            result,
                            operStatus,
                            lookupUrlState.ClientIpAddress);
                    }
                    EventLogger.LogTopLevelExceptionThreadSuccess();
                }
                catch (Exception ex)
                {
                    TraceUtility.TraceCatchException(Tracing.Source, ex);

                    //
                    // Ignore the API Logger exception since it is not fatal.
                    //
                }
            }
            catch (Exception ex)
            {
                operStatus = StatusCodes.Error;
                result = null;

                if (DeliveryServiceHelpers.IsFatalException(ex))
                {
                    throw;
                }
                EventLogger.LogTopLevelExceptionThreadFailure(ex.ToString());
                TraceUtility.TraceCatchException(Tracing.Source, ex);
            }
            finally
            {
                if (lookupUrlState != null && lookupUrlState.Categorizer != null)
                {
                    lookupUrlState.Categorizer.Dispose();
                }

                if (lookupUrlAsyncResult != null)
                {
                    PerfCounters.LookupCategoriesForUrlApiCompleted(lookupUrlAsyncResult.StopWatchTicks);
                }

                TraceUtility.TraceInfo(
                    Tracing.Source,
                    " [{0}] [PERF] EndLookupCategoriesForUrl [{1} ms] [{2} ms] {3} {4}",
                    clientIdGuid.ToString(),
                    lookupUrlAsyncResult == null ? "null" : lookupUrlAsyncResult.StopWatchMilliSec.ToString(),
                    lookupUrlAsyncResult == null ? "null" : lookupUrlAsyncResult.OwnState.DataProviderLatency.ToString(),
                    lookupUrlState == null ? "null" : lookupUrlState.Status.ToString(),
                    result == null ? "null" : result.Count.ToString());
            }

            return result;
        }

        #endregion // Lookup Categories For Url handling

        #region Validators
        internal static uint AreArgumentsValid(
            ClientIdToken clientId,
            LookupRequest[] requests,
            int allowedRequestEntriesCount,
            long clientOverrideStamp)
        {
            if (clientOverrideStamp < 0)
            {
                TraceUtility.TraceWarning(
                    Tracing.Source,
                    "Invalid request: client override stamp is less than zero: {0}",
                    clientOverrideStamp);

                return StatusCodes.InvalidOverrideStamp;
            }

            if(requests == null)
            {
                TraceUtility.TraceWarning(Tracing.Source, "Invalid request: null requests array.");
                return StatusCodes.NullLookupRequest;
            }

            if (requests.Length > allowedRequestEntriesCount)
            {
                TraceUtility.TraceWarning(
                    Tracing.Source,
                    "Invalid request: number of individual requests {0} is more than allowed {1}.",
                    requests.Length,
                    allowedRequestEntriesCount);

                return StatusCodes.TooManyRequests;
            }
            
            uint status = ClientIdToken.IsValidRequest(clientId);

            if (status != StatusCodes.OK)
            {
                return status;
            }

            foreach (LookupRequest lookupRequest in requests)
            {
                status = LookupRequest.IsValidRequest(lookupRequest);

                if (status != StatusCodes.OK)
                {
                    return status;
                }
            }

            return StatusCodes.OK;
        }

        internal static uint AreArgumentsValid(
            ClientIdToken clientId,
            ListingRequest[] requests,
            int allowedRequestEntriesCount,
            ListingToken listingToken)
        {
            if (requests == null)
            {
                TraceUtility.TraceWarning(Tracing.Source, "Invalid request: null requests array.");
                return StatusCodes.NullListingRequest;
            }

            if (requests.Length > allowedRequestEntriesCount)
            {
                TraceUtility.TraceWarning(
                    Tracing.Source,
                    "Invalid request: number of individual requests {0} is more than allowed {1}.",
                    requests.Length,
                    allowedRequestEntriesCount);
                return StatusCodes.TooManyRequests;
            }

            uint status = ListingToken.IsValidRequest(listingToken, requests);

            if (status != StatusCodes.OK)
            {
                return status;
            }

            status = ClientIdToken.IsValidRequest(clientId);

            if (status != StatusCodes.OK)
            {
                return status;
            }

            foreach (ListingRequest listingRequest in requests)
            {
                status = ListingRequest.IsValidRequest(listingRequest);

                if (status != StatusCodes.OK)
                {
                    return status;
                }
            }

            return StatusCodes.OK;
        }

        internal static uint AreArgumentsValid(
            ClientIdToken clientId,
            string url)
        {
            uint status = ClientIdToken.IsValidRequest(clientId);

            if (status != StatusCodes.OK)
            {
                return status;
            }

            if (String.IsNullOrEmpty(url))
            {
                return StatusCodes.NullOrEmptyUrlString;
            }

            return StatusCodes.OK;
        }

        #endregion // Validators

        #region Overrides Logic

        internal static LookupRequest[] AddOverridesRequest(LookupRequest[] lookupRequests, ref long latestOverrideStamp)
        {
            List<LookupRequest> allRequests = new List<LookupRequest>(lookupRequests);

            List<Metadata> overrideHashes = new List<Metadata>();
            SortedList<long, byte[]> overrideListFromBackEnd = MrsDeliveryService.OverrideHashesFromBackEnd;

            long inputStamp = latestOverrideStamp;
            try
            {
                // Get the overrides to add to the request.  We use the Take method so we only get
                // a subset that we can safely put into a response.
                IEnumerable<KeyValuePair<long, byte[]>> overridesToAdd =
                    (from m in overrideListFromBackEnd
                     where m.Key > inputStamp
                     select m)
                    .Take(MrsDeliveryService.MaxOverridesPerPacket);

                // Out of the overrides create a list of Metadata objects that are the hashes for each of the 
                // objects to override.  The Select is just a short hand for a loop over the objects that
                // does a new Metadata.
                overrideHashes.AddRange(overridesToAdd.Select(o => new Metadata("SHA256", o.Value)));

                // We didn't find anything so bail out and just
                // return the same list that was passed in
                if (overrideHashes.Count == 0)
                {
                    return lookupRequests;
                }

                latestOverrideStamp = overridesToAdd.Max(o => o.Key);

                LookupRequest overrideRequest = new LookupRequest(lookupRequests[0].Type, overrideHashes);
                overrideRequest.OverrideRequest = true;

                allRequests.Add(overrideRequest);
            }
            catch (Exception ex)
            {
                TraceUtility.TraceCatchException(Tracing.Source, ex);
                // Swallow the exception to allow processing to continue
            }
            return allRequests.ToArray();
        }

        #endregion // Overrides Logic

        internal List<LookupResponse> BuildCopyOfResponseMatches(List<LookupResponse> result)
        {
            foreach (LookupResponse response in result)
            {
                if (response.Matches != null)
                {
                    List<ReputationObject> copyList = new List<ReputationObject>(response.Matches);
                    response.Matches = copyList;
                }
            }
            return result;
        }

        internal static SimpleReputationData[] CreateSimpleReputationDataFromResponses(DPLookupResponse response)
        {
            if (response == null
                || response.Matches == null
                || response.Matches.Count == 0)
            {
                return new SimpleReputationData[0];
            }

            IEnumerable<SimpleReputationData> simpleReputationData =
                from match in response.Matches
                where match != null
                select (new SimpleReputationData()
                {
                    sha256 = match.Sha256,
                    bloom = 
                        match.Metadatas.ContainsKey(UrlBinaryCodec.MetadataNameBloom) ?
                            match.Metadatas[UrlBinaryCodec.MetadataNameBloom] as byte[] :
                            null,
                    categories = match.Reputations.Select(repu => new InputCategory
                    {
                        categoryId = (uint)repu.Category,
                        confidence = repu.Confidence,
                        pathInheritance = (bool)repu.Metadatas[UrlBinaryCodec.MetadataNamePathInheritance],
                        hostInheritance = (bool)repu.Metadatas[UrlBinaryCodec.MetadataNameHostInheritance]
                    }).ToArray()
                });

            return simpleReputationData.ToArray();
        }
    }
}
